package com.worms.crawler.tactis;

import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.baomidou.mybatisplus.plugins.Page;
import com.kl.common.util.HttpRequestUtil;
import com.kl.common.zookeeper.DistributedLock;
import com.kl.modular.worm.dao.model.WormGrabedPage;
import com.kl.modular.worm.dao.model.WormHomePage;
import com.kl.modular.worm.dao.model.WormPageRegular;
import com.kl.modular.worm.dao.model.WormPendPage;
import com.kl.modular.worm.service.api.IWormGrabedPageService;
import com.kl.modular.worm.service.api.IWormHomePageService;
import com.kl.modular.worm.service.api.IWormPageRegularService;
import com.kl.modular.worm.service.api.IWormPendPageService;
import com.worms.crawler.api.IDefaultCrawler;
import com.worms.crawler.api.IDefaultCrawlerCallback;
import com.worms.crawler.model.CrawlerParam;
import com.worms.http.api.IDefautHttpProxyData;
import com.worms.http.util.IpUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;

/**
 * 基本策略的实现类
 *
 * @author staywood
 * @date 2018/04/20 15:25
 *
 */
public class DefaultCrawler extends CrawlerParam implements IDefaultCrawler{

    private String theadstr = String.valueOf(System.currentTimeMillis() * 8);

    // 中转注入service接口
    private HashMap<String, Object> dataObjMap;

    // 缓存抓取好了的待抓取列表
    protected List<String> newsCachePendPageList = new ArrayList<String>();

    protected IDefaultCrawlerCallback defaultCrawerCallback;

    protected IDefautHttpProxyData defautHttpProxyData;

    protected IWormHomePageService wormHomePageService;

    protected IWormGrabedPageService wormGrabedPageService;

    protected IWormPageRegularService wormPageRegularService;

    protected IWormPendPageService wormPendPageService;

    public DefaultCrawler(){

    }

    /**
     * 爬虫基本配置初始化
     */
    public void init() {
        wormHomePageService = (IWormHomePageService) dataObjMap.get("wormHomePageService");
        wormGrabedPageService = (IWormGrabedPageService) dataObjMap.get("wormGrabedPageService");
        wormPageRegularService = (IWormPageRegularService) dataObjMap.get("wormPageRegularService");
        wormPendPageService = (IWormPendPageService) dataObjMap.get("wormPendPageService");

        // 起始页初始化
        wormHomePageService.addOrUpdate(getHomePageList());
        // 起始页初始化加入带抓取列表
        addGrabList(getHomePageList(), 1);
        // 将匹配页面默认加入种子页面
        this.getSeedList().addAll(getRegularList());
        // 种子初始化
        wormPageRegularService.addOrUpdate(this.getSeedList(), WormPageRegular.REGULAR_SEED);
        this.setSeedList(wormPageRegularService.findTypeList(WormPageRegular.REGULAR_SEED));
        // 匹配页面初始化
        wormPageRegularService.addOrUpdate(this.getRegularList(), WormPageRegular.REGULAR_PAGE);
        this.setRegularList(wormPageRegularService.findTypeList(WormPageRegular.REGULAR_PAGE));
    }

    public void execute() {
        newPendPageRuning();
    }

    /**
     * 获取带抓取列表
     * @return
     */
    public List<WormPendPage> getPendListUrlPage() {
        Page<WormPendPage> page = new Page<WormPendPage>();
        page.setSize(100);
        EntityWrapper<WormPendPage> pendPageEntityWrapper = new EntityWrapper<WormPendPage>();
        pendPageEntityWrapper.eq("status", WormPendPage.STATUS_NORMAL);
        pendPageEntityWrapper.eq("enabled", false);
        pendPageEntityWrapper.orderBy("modify_time", true);
        page = wormPendPageService.selectPage(page, pendPageEntityWrapper);
        if(page != null && page.getRecords() != null){
            for(int i = 0;i < page.getRecords().size();i++){
                page.getRecords().get(i).setQueueTime(new Date());
                page.getRecords().get(i).setStatus(WormPendPage.STATUS_RUNING);
                page.getRecords().get(i).setModifyTime(new Date());
            }
            if(page.getRecords().size() != 0){
                wormPendPageService.updateBatchById(page.getRecords());
            }else{
                // 已经没有待抓取的列表数据了

            }
        }
        return page.getRecords();
    }


    /**
     * 新增已抓取成功页面
     * @param url
     */
    public void addGrabedPageUrl(String url) {
        if(StringUtils.isEmpty(url)){
            return;
        }
        WormGrabedPage pageData = new WormGrabedPage();
        pageData.setPageUrl(url);
        pageData  = wormGrabedPageService.selectOneByEntity(pageData);
        if(pageData != null){
            pageData.setWormEndName(IpUtils.getIp());
            pageData.setWormUpdateTime(new Date());
            wormGrabedPageService.updateById(pageData);
            return;
        }
        WormGrabedPage wormGrabedPage = new WormGrabedPage();
        wormGrabedPage.setStatus(WormGrabedPage.STATUS_WAIT);
        wormGrabedPage.setPageUrl(url);
        wormGrabedPage.setCreateTime(new Date());
        wormGrabedPage.setModifyTime(new Date());
        wormGrabedPage.setWormEndName(IpUtils.getIp());
        wormGrabedPage.setWormUpdateTime(new Date());
        wormGrabedPage.setEnabled(false);
        wormGrabedPageService.insert(wormGrabedPage);
    }

    /**
     * 获取新增带抓取链接
     * @param newGrabList
     */
    public void addGrabList(List<String> newGrabList, int maxNum) {
        if(newGrabList == null || newGrabList.size() == 0){
            return;
        }
        synchronized ("newsCachePendPage"){
            newsCachePendPageList.addAll(newGrabList);
            // 去重
            final List<String> handleNewCachePendPageList = new ArrayList<String>();
            for(String s: newsCachePendPageList){
                if(Collections.frequency(handleNewCachePendPageList, s) < 1) handleNewCachePendPageList.add(s);
            }
            newsCachePendPageList = handleNewCachePendPageList;
        }
    }

    /**
     * 独立线程新增带抓取任务
     *
     */
    public void newPendPageRuning(){
        Thread thread = new Thread(){
            public void run(){
                while(true){
                    List<String> queueList = new ArrayList<String>();
                    synchronized ("newsCachePendPage"){
                        if(newsCachePendPageList.size() == 0){
                            continue;
                        }
                        int toIndex = 100;
                        if(newsCachePendPageList.size() < 100){
                            toIndex = newsCachePendPageList.size();
                        }
                        if(toIndex != 0){
                            List<String> subList = newsCachePendPageList.subList(0, toIndex);
                            for(int i = 0;i<subList.size(); i++){
                                String pendPage = new String(subList.get(i));
                                queueList.add(pendPage);
                            }
                            newsCachePendPageList = newsCachePendPageList.subList(toIndex, newsCachePendPageList.size());
                        }
                    }
                    try{
                        if(queueList.size() != 0){
                            System.out.println("开始加入待抓取同步数据数>>" + queueList.size());
                            wormPendPageService.addOrUpdate(queueList, getCrawlerName());
                            System.out.println("待抓取同步数据完成数>>" + queueList.size());
                        }
                    }catch (Exception e){
                        e.printStackTrace();
                    }
                }
            }
        };
        thread.start();
    }

    /**
     * 获取待抓取链接
     * @return
     */
    public WormPendPage getThreadGrabUrl(){
        synchronized (theadstr){
            if(getGrabUrlList() == null || getGrabUrlList().size() == 0){
                WormPendPage wormPendPage = new WormPendPage();
                wormPendPage.setStatus(WormPendPage.STATUS_NORMAL);
                int count = wormPendPageService.selectCountByEntity(wormPendPage);
                if(count == 0){
                    return null;
                }
                DistributedLock.getLock("pendPageUrl_lock");
                List<WormPendPage> wormPendPageList = getPendListUrlPage();
                DistributedLock.alsoLock("pendPageUrl_lock");
                setGrabUrlList(wormPendPageList);
                return null;
            }
            WormPendPage grabUrl = getGrabUrlList().get(0);
            getGrabUrlList().remove(0);
            return grabUrl;
        }
    }

    /**
     * 从源头开始抓取
     * @return
     */
    public void getHomePageInit(){
        DistributedLock.getLock("grabedHome_lock");
        int times = 60 * 24;
        Date nowDate = new Date();
        List<WormHomePage> homePageList = wormHomePageService.selectListByEntity(new WormHomePage());
        if(homePageList != null && homePageList.size() != 0){
            // 查找指定时间还未重新更新的源头网站
            for(int i = 0;i<homePageList.size();i++){
                WormHomePage homePage = homePageList.get(i);
                if(homePage.getModifyTime() == null){
                    WormGrabedPage grabedPage = new WormGrabedPage();

                }
            }
        }
        DistributedLock.alsoLock("grabedHome_lock");
    }

    public void setDataObjMap(HashMap<String, Object> dataObjMap) {
        this.dataObjMap = dataObjMap;
    }
}
